Spring-boot 集成 kakfa 源码分析

SpringBoot 版本：2.0.0.BUILD-SNAPSHOT

1. 
   

类 KafkaAutoConfiguration

主要功能：自动创建各种bean: KafkaTemplate, messageConverter,producerFactory,consumerFactory等。

@Configuration

@ConditionalOnClass(KafkaTemplate.class)

@EnableConfigurationProperties(KafkaProperties.class)

@Import(KafkaAnnotationDrivenConfiguration.class)

public class KafkaAutoConfiguration {

private final KafkaProperties properties;

private final RecordMessageConverter messageConverter;

public KafkaAutoConfiguration(KafkaProperties properties,

ObjectProvider<RecordMessageConverter> messageConverter) {

this.properties = properties;

this.messageConverter = messageConverter.getIfUnique();

}

@Bean

@ConditionalOnMissingBean(KafkaTemplate.class)

public KafkaTemplate<?, ?> kafkaTemplate(

ProducerFactory<Object, Object> kafkaProducerFactory,

ProducerListener<Object, Object> kafkaProducerListener) {

KafkaTemplate<Object, Object> kafkaTemplate = new KafkaTemplate<>(

kafkaProducerFactory);

if (this.messageConverter != null) {

kafkaTemplate.setMessageConverter(this.messageConverter);

}

kafkaTemplate.setProducerListener(kafkaProducerListener);

kafkaTemplate.setDefaultTopic(this.properties.getTemplate().getDefaultTopic());

return kafkaTemplate;

}

@Bean

@ConditionalOnMissingBean(ProducerListener.class)

public ProducerListener<Object, Object> kafkaProducerListener() {

return new LoggingProducerListener<>();

}

@Bean

@ConditionalOnMissingBean(ConsumerFactory.class)

public ConsumerFactory<?, ?> kafkaConsumerFactory() {

return new DefaultKafkaConsumerFactory<>(

this.properties.buildConsumerProperties());

}

@Bean

@ConditionalOnMissingBean(ProducerFactory.class)

public ProducerFactory<?, ?> kafkaProducerFactory() {

DefaultKafkaProducerFactory<?, ?> factory = new DefaultKafkaProducerFactory<>(

this.properties.buildProducerProperties());

String transactionIdPrefix = this.properties.getProducer()

.getTransactionIdPrefix();

if (transactionIdPrefix != null) {

factory.setTransactionIdPrefix(transactionIdPrefix);

}

return factory;

}

@Bean

@ConditionalOnProperty(name = "spring.kafka.producer.transaction-id-prefix")

@ConditionalOnMissingBean

public KafkaTransactionManager<?, ?> kafkaTransactionManager(

ProducerFactory<?, ?> producerFactory) {

return new KafkaTransactionManager<>(producerFactory);

}

@Bean

@ConditionalOnProperty(name = "spring.kafka.jaas.enabled")

@ConditionalOnMissingBean

public KafkaJaasLoginModuleInitializer kafkaJaasInitializer() throws IOException {

KafkaJaasLoginModuleInitializer jaas = new KafkaJaasLoginModuleInitializer();

Jaas jaasProperties = this.properties.getJaas();

if (jaasProperties.getControlFlag() != null) {

jaas.setControlFlag(jaasProperties.getControlFlag());

}

if (jaasProperties.getLoginModule() != null) {

jaas.setLoginModule(jaasProperties.getLoginModule());

}

jaas.setOptions(jaasProperties.getOptions());

return jaas;

}

@Bean

@ConditionalOnMissingBean(KafkaAdmin.class)

public KafkaAdmin kafkaAdmin() {

KafkaAdmin kafkaAdmin = new KafkaAdmin(this.properties.buildAdminProperties());

kafkaAdmin.setFatalIfBrokerNotAvailable(this.properties.getAdmin().isFailFast());

return kafkaAdmin;

}

}

类 KafkaAnnotationDrivenConfiguration

主要功能：自动创建bean ：kafkaListenerContainerFactory。

/**

* Configuration for Kafka annotation-driven support.

*

* @author Gary Russell

* @author Eddú Meléndez

* @since 1.5.0

*/

@Configuration

@ConditionalOnClass(EnableKafka.class)

class KafkaAnnotationDrivenConfiguration {

private final KafkaProperties properties;

private final RecordMessageConverter messageConverter;

private final KafkaTemplate<Object, Object> kafkaTemplate;

KafkaAnnotationDrivenConfiguration(KafkaProperties properties,

ObjectProvider<RecordMessageConverter> messageConverter,

ObjectProvider<KafkaTemplate<Object, Object>> kafkaTemplate) {

this.properties = properties;

this.messageConverter = messageConverter.getIfUnique();

this.kafkaTemplate = kafkaTemplate.getIfUnique();

}

@Bean

@ConditionalOnMissingBean

public ConcurrentKafkaListenerContainerFactoryConfigurer kafkaListenerContainerFactoryConfigurer() {

ConcurrentKafkaListenerContainerFactoryConfigurer configurer = new ConcurrentKafkaListenerContainerFactoryConfigurer();

configurer.setKafkaProperties(this.properties);

configurer.setMessageConverter(this.messageConverter);

configurer.setReplyTemplate(this.kafkaTemplate);

return configurer;

}

@Bean

@ConditionalOnMissingBean(name = "kafkaListenerContainerFactory")

public ConcurrentKafkaListenerContainerFactory<?, ?> kafkaListenerContainerFactory(

ConcurrentKafkaListenerContainerFactoryConfigurer configurer,

ConsumerFactory<Object, Object> kafkaConsumerFactory) {

ConcurrentKafkaListenerContainerFactory<Object, Object> factory = new ConcurrentKafkaListenerContainerFactory<>();

configurer.configure(factory, kafkaConsumerFactory);

return factory;

}

@EnableKafka

@ConditionalOnMissingBean(name = KafkaListenerConfigUtils.KAFKA_LISTENER_ANNOTATION_PROCESSOR_BEAN_NAME)

protected static class EnableKafkaConfiguration {

}

}

源码解析：

(1) ConcurrentKafkaListenerContainerFactoryConfigurer 这个类是用来配置kafkaListenerContainerFactory。

public void configure(

ConcurrentKafkaListenerContainerFactory<Object, Object> listenerContainerFactory,

ConsumerFactory<Object, Object> consumerFactory) {

listenerContainerFactory.setConsumerFactory(consumerFactory);

if (this.messageConverter != null) {

listenerContainerFactory.setMessageConverter(this.messageConverter);

}

if (this.replyTemplate != null) {

listenerContainerFactory.setReplyTemplate(this.replyTemplate);

}

Listener container = this.properties.getListener();

ContainerProperties containerProperties = listenerContainerFactory

.getContainerProperties();

if (container.getAckMode() != null) {

containerProperties.setAckMode(container.getAckMode());

}

if (container.getAckCount() != null) {

containerProperties.setAckCount(container.getAckCount());

}

if (container.getAckTime() != null) {

containerProperties.setAckTime(container.getAckTime().toMillis());

}

if (container.getPollTimeout() != null) {

containerProperties.setPollTimeout(container.getPollTimeout().toMillis());

}

if (container.getConcurrency() != null) {

listenerContainerFactory.setConcurrency(container.getConcurrency());

}

if (container.getType() == Listener.Type.BATCH) {

listenerContainerFactory.setBatchListener(true);

}

}

（2）ConcurrentKafkaListenerContainerFactory

主要功能：依据给定的kafka监听目标endPoint 生成MessageListenerContainer的实例。该实例全权负责对给定的监听目标endPoint的所有操作：循环从目标中取数据 --> 处理数据：利用实例中依据配置生成messageListener 的onMessage(...)方法处理数据。该实例是可以设置并发的，即concurrency属性设为N，则生成N个Consumer监听线程从该监听目标中取数据/处理。

public class ConcurrentKafkaListenerContainerFactory<K, V>

extends AbstractKafkaListenerContainerFactory<ConcurrentMessageListenerContainer<K, V>, K, V> {

private Integer concurrency;

/**

* Specify the container concurrency.

* @param concurrency the number of consumers to create.

* @see ConcurrentMessageListenerContainer#setConcurrency(int)

*/

public void setConcurrency(Integer concurrency) {

this.concurrency = concurrency;

}

@Override

protected ConcurrentMessageListenerContainer<K, V> createContainerInstance(KafkaListenerEndpoint endpoint) {

Collection<TopicPartitionInitialOffset> topicPartitions = endpoint.getTopicPartitions();

if (!topicPartitions.isEmpty()) {

ContainerProperties properties = new ContainerProperties(

topicPartitions.toArray(new TopicPartitionInitialOffset[topicPartitions.size()]));

return new ConcurrentMessageListenerContainer<K, V>(getConsumerFactory(), properties);

}

else {

Collection<String> topics = endpoint.getTopics();

if (!topics.isEmpty()) {

ContainerProperties properties = new ContainerProperties(topics.toArray(new String[topics.size()]));

return new ConcurrentMessageListenerContainer<K, V>(getConsumerFactory(), properties);

}

else {

ContainerProperties properties = new ContainerProperties(endpoint.getTopicPattern());

return new ConcurrentMessageListenerContainer<K, V>(getConsumerFactory(), properties);

}

}

}